

<!DOCTYPE html>
<html lang="zh-CN" data-default-color-scheme=auto>



<head>
  <meta charset="UTF-8">
  <link rel="apple-touch-icon" sizes="76x76" href="/img/fluid.png">
  <link rel="icon" href="/img/fluid.png">
  <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=5.0, shrink-to-fit=no">
  <meta http-equiv="x-ua-compatible" content="ie=edge">
  
  <meta name="theme-color" content="#2f4154">
  <meta name="author" content="John Doe">
  <meta name="keywords" content="">
  
    <meta name="description" content="@[toc] 参考资料 视频资料 运行环境  Windows10 JDK8 IDEA 2021.6 专业版 Hadoop3.1.3 CentOS7 3个Hadoop完全分布式集群节点  一、准备 HDFS Java API 环境 1.1 在windows系统中准备Hadoop环境Hadoop3.1.3 官方下载地址：点击下载，下载后解压。 hadoop主要基于linux编写，这个winutil.e">
<meta property="og:type" content="article">
<meta property="og:title" content="Hadoop _ HDFS 学习笔记（二）HDFS Java API 环境搭建 _ Java操作HDFS文件系统 _ 多案例">
<meta property="og:url" content="http://example.com/2022/01/18/Hadoop%20_%20HDFS%20%E5%AD%A6%E4%B9%A0%E7%AC%94%E8%AE%B0%EF%BC%88%E4%BA%8C%EF%BC%89HDFS%20Java%20API%20%E7%8E%AF%E5%A2%83%E6%90%AD%E5%BB%BA%20_%20Java%E6%93%8D%E4%BD%9CHDFS%E6%96%87%E4%BB%B6%E7%B3%BB%E7%BB%9F%20_%20%E5%A4%9A%E6%A1%88%E4%BE%8B/index.html">
<meta property="og:site_name" content="Hexo">
<meta property="og:description" content="@[toc] 参考资料 视频资料 运行环境  Windows10 JDK8 IDEA 2021.6 专业版 Hadoop3.1.3 CentOS7 3个Hadoop完全分布式集群节点  一、准备 HDFS Java API 环境 1.1 在windows系统中准备Hadoop环境Hadoop3.1.3 官方下载地址：点击下载，下载后解压。 hadoop主要基于linux编写，这个winutil.e">
<meta property="og:locale" content="zh_CN">
<meta property="og:image" content="https://img-blog.csdnimg.cn/216a86ee46f240b89f94057499070a86.png#pic_center">
<meta property="og:image" content="https://img-blog.csdnimg.cn/78c96cd8651c40ca8f68ade8ed255a5b.png#pic_center">
<meta property="og:image" content="https://img-blog.csdnimg.cn/49fb0fd89e514ad7b7a7ed221430c24c.png#pic_center">
<meta property="og:image" content="https://img-blog.csdnimg.cn/bc1a00fdba204249879902b08c9012a7.png#pic_center">
<meta property="og:image" content="https://img-blog.csdnimg.cn/c29da3ae68c44948b2bf80a05fb34db5.png">
<meta property="og:image" content="https://img-blog.csdnimg.cn/5555ef095f7d415990e69c92e1f9d8bf.png">
<meta property="og:image" content="https://img-blog.csdnimg.cn/bcd003b30810439da3e4b235eaddcdc2.png">
<meta property="og:image" content="https://img-blog.csdnimg.cn/3156ce6798a74e69b38b5c01df68d4b6.png">
<meta property="article:published_time" content="2022-01-18T10:02:06.000Z">
<meta property="article:modified_time" content="2022-08-22T15:43:56.004Z">
<meta property="article:author" content="John Doe">
<meta property="article:tag" content="HDFS">
<meta name="twitter:card" content="summary_large_image">
<meta name="twitter:image" content="https://img-blog.csdnimg.cn/216a86ee46f240b89f94057499070a86.png#pic_center">
  
  
  
  <title>Hadoop _ HDFS 学习笔记（二）HDFS Java API 环境搭建 _ Java操作HDFS文件系统 _ 多案例 - Hexo</title>

  <link  rel="stylesheet" href="https://lib.baomitu.com/twitter-bootstrap/4.6.1/css/bootstrap.min.css" />



  <link  rel="stylesheet" href="https://lib.baomitu.com/github-markdown-css/4.0.0/github-markdown.min.css" />

  <link  rel="stylesheet" href="https://lib.baomitu.com/hint.css/2.7.0/hint.min.css" />

  <link  rel="stylesheet" href="https://lib.baomitu.com/fancybox/3.5.7/jquery.fancybox.min.css" />



<!-- 主题依赖的图标库，不要自行修改 -->
<!-- Do not modify the link that theme dependent icons -->

<link rel="stylesheet" href="//at.alicdn.com/t/font_1749284_hj8rtnfg7um.css">



<link rel="stylesheet" href="//at.alicdn.com/t/font_1736178_lbnruvf0jn.css">


<link  rel="stylesheet" href="/css/main.css" />


  <link id="highlight-css" rel="stylesheet" href="/css/highlight.css" />
  
    <link id="highlight-css-dark" rel="stylesheet" href="/css/highlight-dark.css" />
  




  <script id="fluid-configs">
    var Fluid = window.Fluid || {};
    Fluid.ctx = Object.assign({}, Fluid.ctx)
    var CONFIG = {"hostname":"example.com","root":"/","version":"1.9.2","typing":{"enable":true,"typeSpeed":70,"cursorChar":"_","loop":false,"scope":[]},"anchorjs":{"enable":true,"element":"h1,h2,h3,h4,h5,h6","placement":"left","visible":"hover","icon":""},"progressbar":{"enable":true,"height_px":3,"color":"#29d","options":{"showSpinner":false,"trickleSpeed":100}},"code_language":{"enable":true,"default":"TEXT"},"copy_btn":true,"image_caption":{"enable":true},"image_zoom":{"enable":true,"img_url_replace":["",""]},"toc":{"enable":true,"placement":"right","headingSelector":"h1,h2,h3,h4,h5,h6","collapseDepth":0},"lazyload":{"enable":true,"loading_img":"/img/loading.gif","onlypost":false,"offset_factor":2},"web_analytics":{"enable":false,"follow_dnt":true,"baidu":null,"google":null,"gtag":null,"tencent":{"sid":null,"cid":null},"woyaola":null,"cnzz":null,"leancloud":{"app_id":null,"app_key":null,"server_url":null,"path":"window.location.pathname","ignore_local":false}},"search_path":"/local-search.xml"};

    if (CONFIG.web_analytics.follow_dnt) {
      var dntVal = navigator.doNotTrack || window.doNotTrack || navigator.msDoNotTrack;
      Fluid.ctx.dnt = dntVal && (dntVal.startsWith('1') || dntVal.startsWith('yes') || dntVal.startsWith('on'));
    }
  </script>
  <script  src="/js/utils.js" ></script>
  <script  src="/js/color-schema.js" ></script>
  


  
<meta name="generator" content="Hexo 6.2.0"></head>


<body>
  

  <header>
    

<div class="header-inner" style="height: 70vh;">
  <nav id="navbar" class="navbar fixed-top  navbar-expand-lg navbar-dark scrolling-navbar">
  <div class="container">
    <a class="navbar-brand" href="/">
      <strong>尤 Ni&#39;s Blog</strong>
    </a>

    <button id="navbar-toggler-btn" class="navbar-toggler" type="button" data-toggle="collapse"
            data-target="#navbarSupportedContent"
            aria-controls="navbarSupportedContent" aria-expanded="false" aria-label="Toggle navigation">
      <div class="animated-icon"><span></span><span></span><span></span></div>
    </button>

    <!-- Collapsible content -->
    <div class="collapse navbar-collapse" id="navbarSupportedContent">
      <ul class="navbar-nav ml-auto text-center">
        
          
          
          
          
            <li class="nav-item">
              <a class="nav-link" href="/">
                <i class="iconfont icon-home-fill"></i>
                首页
              </a>
            </li>
          
        
          
          
          
          
            <li class="nav-item">
              <a class="nav-link" href="/archives/">
                <i class="iconfont icon-archive-fill"></i>
                归档
              </a>
            </li>
          
        
          
          
          
          
            <li class="nav-item">
              <a class="nav-link" href="/categories/">
                <i class="iconfont icon-category-fill"></i>
                分类
              </a>
            </li>
          
        
          
          
          
          
            <li class="nav-item">
              <a class="nav-link" href="/tags/">
                <i class="iconfont icon-tags-fill"></i>
                标签
              </a>
            </li>
          
        
          
          
          
          
            <li class="nav-item">
              <a class="nav-link" href="/about/">
                <i class="iconfont icon-user-fill"></i>
                关于
              </a>
            </li>
          
        
        
          <li class="nav-item" id="search-btn">
            <a class="nav-link" target="_self" href="javascript:;" data-toggle="modal" data-target="#modalSearch" aria-label="Search">
              &nbsp;<i class="iconfont icon-search"></i>&nbsp;
            </a>
          </li>
          
        
        
          <li class="nav-item" id="color-toggle-btn">
            <a class="nav-link" target="_self" href="javascript:;" aria-label="Color Toggle">&nbsp;<i
                class="iconfont icon-dark" id="color-toggle-icon"></i>&nbsp;</a>
          </li>
        
      </ul>
    </div>
  </div>
</nav>

  

<div id="banner" class="banner" parallax=true
     style="background: url('/img/default.png') no-repeat center center; background-size: cover;">
  <div class="full-bg-img">
    <div class="mask flex-center" style="background-color: rgba(0, 0, 0, 0.3)">
      <div class="banner-text text-center fade-in-up">
        <div class="h2">
          
            <span id="subtitle" data-typed-text="Hadoop _ HDFS 学习笔记（二）HDFS Java API 环境搭建 _ Java操作HDFS文件系统 _ 多案例"></span>
          
        </div>

        
          
  <div class="mt-3">
    
      <span class="post-meta mr-2">
        <i class="iconfont icon-author" aria-hidden="true"></i>
        John Doe
      </span>
    
    
      <span class="post-meta">
        <i class="iconfont icon-date-fill" aria-hidden="true"></i>
        <time datetime="2022-01-18 18:02" pubdate>
          2022年1月18日 晚上
        </time>
      </span>
    
  </div>

  <div class="mt-1">
    
      <span class="post-meta mr-2">
        <i class="iconfont icon-chart"></i>
        
          7.8k 字
        
      </span>
    

    
      <span class="post-meta mr-2">
        <i class="iconfont icon-clock-fill"></i>
        
        
        
          66 分钟
        
      </span>
    

    
    
  </div>


        
      </div>

      
    </div>
  </div>
</div>

</div>

  </header>

  <main>
    
      

<div class="container-fluid nopadding-x">
  <div class="row nomargin-x">
    <div class="side-col d-none d-lg-block col-lg-2">
      

    </div>

    <div class="col-lg-8 nopadding-x-md">
      <div class="container nopadding-x-md" id="board-ctn">
        <div id="board">
          <article class="post-content mx-auto">
            <!-- SEO header -->
            <h1 style="display: none">Hadoop _ HDFS 学习笔记（二）HDFS Java API 环境搭建 _ Java操作HDFS文件系统 _ 多案例</h1>
            
              <p class="note note-info">
                
                  
                    本文最后更新于：1 小时前
                  
                
              </p>
            
            
              <div class="markdown-body">
                
                <p>@[toc]</p>
<h1 id="参考资料"><a href="#参考资料" class="headerlink" title="参考资料"></a>参考资料</h1><hr>
<p><a target="_blank" rel="noopener" href="https://www.bilibili.com/video/BV1Qp4y1n7EN?p=46">视频资料</a></p>
<h1 id="运行环境"><a href="#运行环境" class="headerlink" title="运行环境"></a>运行环境</h1><hr>
<ul>
<li>Windows10</li>
<li>JDK8</li>
<li>IDEA 2021.6 专业版</li>
<li>Hadoop3.1.3</li>
<li>CentOS7</li>
<li>3个Hadoop完全分布式集群节点</li>
</ul>
<h1 id="一、准备-HDFS-Java-API-环境"><a href="#一、准备-HDFS-Java-API-环境" class="headerlink" title="一、准备 HDFS Java API 环境"></a>一、准备 HDFS Java API 环境</h1><hr>
<h2 id="1-1-在windows系统中准备Hadoop环境"><a href="#1-1-在windows系统中准备Hadoop环境" class="headerlink" title="1.1 在windows系统中准备Hadoop环境"></a>1.1 在windows系统中准备Hadoop环境</h2><p>Hadoop3.1.3 官方下载地址：<a target="_blank" rel="noopener" href="https://archive.apache.org/dist/hadoop/common/hadoop-3.1.3/hadoop-3.1.3.tar.gz">点击下载</a>，下载后解压。</p>
<p>hadoop主要基于linux编写，这个winutil.exe主要用于模拟Linux系统下的目录环境。因此hadoop放在windows下运行的时候，需要这个辅助程序才能运行</p>
<p>这些环境软件下载到hadoop目录的bin文件夹：<a target="_blank" rel="noopener" href="https://gitee.com/ccuni/hadoop-learn/tree/master/hadoop3.1.3-win10-environment">点击下载</a></p>
<p><img src="https://img-blog.csdnimg.cn/216a86ee46f240b89f94057499070a86.png#pic_center" srcset="/img/loading.gif" lazyload alt="在这里插入图片描述"></p>
<p>之后在配置Hadoop的环境变量，跟Linux环境里一样，先配置HADOOP_HOME，再配置系统变量PATH</p>
<p>图形化操作可以右键此电脑，点击属性，然后点击高级系统设置，环境变量</p>
<p><img src="https://img-blog.csdnimg.cn/78c96cd8651c40ca8f68ade8ed255a5b.png#pic_center" srcset="/img/loading.gif" lazyload alt="在这里插入图片描述"></p>
<p><img src="https://img-blog.csdnimg.cn/49fb0fd89e514ad7b7a7ed221430c24c.png#pic_center" srcset="/img/loading.gif" lazyload alt="在这里插入图片描述"></p>
<p>配置好后，在cmd里查看版本</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><code class="hljs shell">hadoop version<br></code></pre></td></tr></table></figure>

<p><img src="https://img-blog.csdnimg.cn/bc1a00fdba204249879902b08c9012a7.png#pic_center" srcset="/img/loading.gif" lazyload alt="在这里插入图片描述"><br>配置完毕.</p>
<h2 id="1-2-使本机连接集群节点"><a href="#1-2-使本机连接集群节点" class="headerlink" title="1.2 使本机连接集群节点"></a>1.2 使本机连接集群节点</h2><h3 id="1-2-1-域名映射"><a href="#1-2-1-域名映射" class="headerlink" title="1.2.1 域名映射"></a>1.2.1 域名映射</h3><p>在本机配置域名映射，这样就可以直接通过主机名连接到对应的ID</p>
<p>windows10配置域名映射，位置在：</p>
<figure class="highlight moonscript"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><code class="hljs moonscript"><span class="hljs-name">C</span>:\WINDOWS\System32\drivers\etc\<br></code></pre></td></tr></table></figure>
<p><img src="https://img-blog.csdnimg.cn/c29da3ae68c44948b2bf80a05fb34db5.png" srcset="/img/loading.gif" lazyload alt="在这里插入图片描述"></p>
<p>hosts的配置内容：<br><img src="https://img-blog.csdnimg.cn/5555ef095f7d415990e69c92e1f9d8bf.png" srcset="/img/loading.gif" lazyload alt="在这里插入图片描述"><br>如图：以 IP + 空格 + 主机名 的形式进行配置</p>
<p>IP的话如果是本机的虚拟机则可以在虚拟机里通过命令查看</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><code class="hljs shell">ifconfig<br></code></pre></td></tr></table></figure>
<p>显示的信息中，选择eth开头的信息，比如在Vmware里是eth33，而在docker容器里就是eth0<br><img src="https://img-blog.csdnimg.cn/bcd003b30810439da3e4b235eaddcdc2.png" srcset="/img/loading.gif" lazyload alt="在这里插入图片描述"></p>
<h3 id="1-2-2-路由转发"><a href="#1-2-2-路由转发" class="headerlink" title="1.2.2 路由转发"></a>1.2.2 路由转发</h3><p>由于笔者是在虚拟机里使用docker容器搭建的三个节点，那么则有：</p>
<ul>
<li>虚拟机能和自身里的docker容器相连</li>
<li>本机能和虚拟机相连</li>
</ul>
<p>若要本机直接和虚拟机里的docker容器相连，就要进行域名转发，可参考这篇博文：<a target="_blank" rel="noopener" href="https://blog.csdn.net/Unirithe/article/details/121668750">Docker | 使用宿主机ping通虚拟机里的docker容器 | 路由转发</a></p>
<p>配置完成后，可以直接在主机ping通集群节点，这样就可以在之后的本机程序里连接集群的HDFS了。</p>
<p><img src="https://img-blog.csdnimg.cn/3156ce6798a74e69b38b5c01df68d4b6.png" srcset="/img/loading.gif" lazyload alt="在这里插入图片描述"></p>
<h2 id="1-3-使用-IDEA-创建-Maven-项目"><a href="#1-3-使用-IDEA-创建-Maven-项目" class="headerlink" title="1.3 使用 IDEA 创建 Maven 项目"></a>1.3 使用 IDEA 创建 Maven 项目</h2><p><strong>项目结构：</strong><br>![在这里插入图片描述](<a target="_blank" rel="noopener" href="https://img-blog.csdnimg.cn/19fe83515ce04e098c4ea4bd942b4e86.png#">https://img-blog.csdnimg.cn/19fe83515ce04e098c4ea4bd942b4e86.png#</a> &#x3D;60%x)</p>
<p><code>pom.xml</code> 配置 Maven项目的依赖</p>
<figure class="highlight xml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br></pre></td><td class="code"><pre><code class="hljs xml"><span class="hljs-meta">&lt;?xml version=<span class="hljs-string">&quot;1.0&quot;</span> encoding=<span class="hljs-string">&quot;UTF-8&quot;</span>?&gt;</span><br><span class="hljs-tag">&lt;<span class="hljs-name">project</span> <span class="hljs-attr">xmlns</span>=<span class="hljs-string">&quot;http://maven.apache.org/POM/4.0.0&quot;</span></span><br><span class="hljs-tag">         <span class="hljs-attr">xmlns:xsi</span>=<span class="hljs-string">&quot;http://www.w3.org/2001/XMLSchema-instance&quot;</span></span><br><span class="hljs-tag">         <span class="hljs-attr">xsi:schemaLocation</span>=<span class="hljs-string">&quot;http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd&quot;</span>&gt;</span><br>    <span class="hljs-tag">&lt;<span class="hljs-name">modelVersion</span>&gt;</span>4.0.0<span class="hljs-tag">&lt;/<span class="hljs-name">modelVersion</span>&gt;</span><br><br>    <span class="hljs-tag">&lt;<span class="hljs-name">groupId</span>&gt;</span>com.uni<span class="hljs-tag">&lt;/<span class="hljs-name">groupId</span>&gt;</span><br>    <span class="hljs-tag">&lt;<span class="hljs-name">artifactId</span>&gt;</span>HDFSLearn<span class="hljs-tag">&lt;/<span class="hljs-name">artifactId</span>&gt;</span><br>    <span class="hljs-tag">&lt;<span class="hljs-name">version</span>&gt;</span>1.0-SNAPSHOT<span class="hljs-tag">&lt;/<span class="hljs-name">version</span>&gt;</span><br><br>    <span class="hljs-tag">&lt;<span class="hljs-name">properties</span>&gt;</span><br>        <span class="hljs-tag">&lt;<span class="hljs-name">maven.compiler.source</span>&gt;</span>8<span class="hljs-tag">&lt;/<span class="hljs-name">maven.compiler.source</span>&gt;</span><br>        <span class="hljs-tag">&lt;<span class="hljs-name">maven.compiler.target</span>&gt;</span>8<span class="hljs-tag">&lt;/<span class="hljs-name">maven.compiler.target</span>&gt;</span><br>    <span class="hljs-tag">&lt;/<span class="hljs-name">properties</span>&gt;</span><br><br>    <span class="hljs-tag">&lt;<span class="hljs-name">dependencies</span>&gt;</span><br>        <span class="hljs-tag">&lt;<span class="hljs-name">dependency</span>&gt;</span><br>            <span class="hljs-tag">&lt;<span class="hljs-name">groupId</span>&gt;</span>org.apache.hadoop<span class="hljs-tag">&lt;/<span class="hljs-name">groupId</span>&gt;</span><br>            <span class="hljs-tag">&lt;<span class="hljs-name">artifactId</span>&gt;</span>hadoop-client<span class="hljs-tag">&lt;/<span class="hljs-name">artifactId</span>&gt;</span><br>            <span class="hljs-tag">&lt;<span class="hljs-name">version</span>&gt;</span>3.1.3<span class="hljs-tag">&lt;/<span class="hljs-name">version</span>&gt;</span><br>        <span class="hljs-tag">&lt;/<span class="hljs-name">dependency</span>&gt;</span><br>        <span class="hljs-tag">&lt;<span class="hljs-name">dependency</span>&gt;</span><br>            <span class="hljs-tag">&lt;<span class="hljs-name">groupId</span>&gt;</span>junit<span class="hljs-tag">&lt;/<span class="hljs-name">groupId</span>&gt;</span><br>            <span class="hljs-tag">&lt;<span class="hljs-name">artifactId</span>&gt;</span>junit<span class="hljs-tag">&lt;/<span class="hljs-name">artifactId</span>&gt;</span><br>            <span class="hljs-tag">&lt;<span class="hljs-name">version</span>&gt;</span>4.12<span class="hljs-tag">&lt;/<span class="hljs-name">version</span>&gt;</span><br>        <span class="hljs-tag">&lt;/<span class="hljs-name">dependency</span>&gt;</span><br>        <span class="hljs-tag">&lt;<span class="hljs-name">dependency</span>&gt;</span><br>            <span class="hljs-tag">&lt;<span class="hljs-name">groupId</span>&gt;</span>org.slf4j<span class="hljs-tag">&lt;/<span class="hljs-name">groupId</span>&gt;</span><br>            <span class="hljs-tag">&lt;<span class="hljs-name">artifactId</span>&gt;</span>slf4j-log4j12<span class="hljs-tag">&lt;/<span class="hljs-name">artifactId</span>&gt;</span><br>            <span class="hljs-tag">&lt;<span class="hljs-name">version</span>&gt;</span>1.7.30<span class="hljs-tag">&lt;/<span class="hljs-name">version</span>&gt;</span><br>        <span class="hljs-tag">&lt;/<span class="hljs-name">dependency</span>&gt;</span><br>    <span class="hljs-tag">&lt;/<span class="hljs-name">dependencies</span>&gt;</span><br><span class="hljs-tag">&lt;/<span class="hljs-name">project</span>&gt;</span><br></code></pre></td></tr></table></figure>

<p><code>log4j.properties</code> 配置 log4j日志文件的输出信息</p>
<figure class="highlight properties"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><code class="hljs properties"><span class="hljs-attr">log4j.rootLogger</span>=<span class="hljs-string">INFO, stdout  </span><br><span class="hljs-attr">log4j.appender.stdout</span>=<span class="hljs-string">org.apache.log4j.ConsoleAppender  </span><br><span class="hljs-attr">log4j.appender.stdout.layout</span>=<span class="hljs-string">org.apache.log4j.PatternLayout  </span><br><span class="hljs-attr">log4j.appender.stdout.layout.ConversionPattern</span>=<span class="hljs-string">%d %p [%c] - %m%n  </span><br><span class="hljs-attr">log4j.appender.logfile</span>=<span class="hljs-string">org.apache.log4j.FileAppender  </span><br><span class="hljs-attr">log4j.appender.logfile.File</span>=<span class="hljs-string">target/spring.log  </span><br><span class="hljs-attr">log4j.appender.logfile.layout</span>=<span class="hljs-string">org.apache.log4j.PatternLayout  </span><br><span class="hljs-attr">log4j.appender.logfile.layout.ConversionPattern</span>=<span class="hljs-string">%d %p [%c] - %m%n</span><br></code></pre></td></tr></table></figure>

<p>至此，IDEA项目搭建完毕。</p>
<h1 id="二、HDFS-Java-API-操作案例"><a href="#二、HDFS-Java-API-操作案例" class="headerlink" title="二、HDFS Java API 操作案例"></a>二、HDFS Java API 操作案例</h1><hr>
<h2 id="2-1-创建文件夹"><a href="#2-1-创建文件夹" class="headerlink" title="2.1 创建文件夹"></a>2.1 创建文件夹</h2><p><code>org.apache.hadoop.fs.FileSystem</code> 源码中的相关方法：</p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><code class="hljs java"><span class="hljs-keyword">public</span> <span class="hljs-keyword">static</span> <span class="hljs-type">boolean</span> <span class="hljs-title function_">mkdirs</span><span class="hljs-params">(FileSystem fs, Path dir, FsPermission permission)</span><br><span class="hljs-keyword">public</span> <span class="hljs-keyword">abstract</span> <span class="hljs-type">boolean</span> <span class="hljs-title function_">mkdirs</span><span class="hljs-params">(Path f, FsPermission permission)</span> <span class="hljs-keyword">throws</span> IOException<br><span class="hljs-keyword">public</span> <span class="hljs-type">boolean</span> <span class="hljs-title function_">mkdirs</span><span class="hljs-params">(Path f)</span> <span class="hljs-keyword">throws</span> IOException<br></code></pre></td></tr></table></figure>
<p><strong>参数说明：</strong></p>
<table>
<thead>
<tr>
<th align="left">参数</th>
<th align="left">描述</th>
</tr>
</thead>
<tbody><tr>
<td align="left"><code>Path f</code></td>
<td align="left">HDFS创建的文件目录</td>
</tr>
<tr>
<td align="left"><code>FsPermission permission</code></td>
<td align="left">当前用户的权限级别</td>
</tr>
</tbody></table>
<p><strong>测试代码：</strong></p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br></pre></td><td class="code"><pre><code class="hljs java"><span class="hljs-keyword">public</span> <span class="hljs-keyword">class</span> <span class="hljs-title class_">HdfsClient</span> &#123;<br>    <span class="hljs-keyword">private</span> <span class="hljs-keyword">static</span> Configuration configuration;<br>    <span class="hljs-keyword">private</span> <span class="hljs-keyword">static</span> <span class="hljs-type">String</span> <span class="hljs-variable">HDFS_PATH</span> <span class="hljs-operator">=</span> <span class="hljs-string">&quot;hdfs://hadoop101:8020&quot;</span>; <span class="hljs-comment">// HDFS 连接地址</span><br>    <span class="hljs-keyword">private</span> <span class="hljs-keyword">static</span> <span class="hljs-type">String</span> <span class="hljs-variable">HDFS_USER</span> <span class="hljs-operator">=</span> <span class="hljs-string">&quot;root&quot;</span>;                  <span class="hljs-comment">// HDFS 连接对象</span><br>    <span class="hljs-keyword">private</span> <span class="hljs-keyword">static</span> FileSystem fs;                              <span class="hljs-comment">// HDFS 文件操作对象</span><br>    <span class="hljs-comment">/* 单例模式 创建连接 **/</span> <br>    <span class="hljs-keyword">static</span> &#123; configuration = <span class="hljs-keyword">new</span> <span class="hljs-title class_">Configuration</span>(); &#125;<br>    <span class="hljs-keyword">public</span> FileSystem <span class="hljs-title function_">getFileSystem</span><span class="hljs-params">()</span>&#123;<br>        <span class="hljs-keyword">try</span>&#123;<br>            <span class="hljs-keyword">if</span>(fs == <span class="hljs-literal">null</span>)<br>                fs = FileSystem.get(<span class="hljs-keyword">new</span> <span class="hljs-title class_">URI</span>(HDFS_PATH), configuration, HDFS_USER);<br>        &#125; <span class="hljs-keyword">catch</span> (Exception e) &#123;<br>            e.printStackTrace();<br>        &#125;<br>        <span class="hljs-keyword">return</span> fs;<br>    &#125;<br>    <span class="hljs-meta">@Test</span><br>    <span class="hljs-keyword">public</span> <span class="hljs-keyword">void</span> <span class="hljs-title function_">testMkdir</span><span class="hljs-params">()</span> <span class="hljs-keyword">throws</span> IOException&#123;<br>        <span class="hljs-type">FileSystem</span> <span class="hljs-variable">fs</span> <span class="hljs-operator">=</span> getFileSystem();<br>        fs.mkdirs(<span class="hljs-keyword">new</span> <span class="hljs-title class_">Path</span>(<span class="hljs-string">&quot;/uni/testMkdir/&quot;</span>));<br>        fs.close();<br>    &#125;<br>&#125;<br></code></pre></td></tr></table></figure>

<h2 id="2-2-上传文件"><a href="#2-2-上传文件" class="headerlink" title="2.2 上传文件"></a>2.2 上传文件</h2><p><code>org.apache.hadoop.fs.FileSystem</code> 源码中的相关方法：</p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><code class="hljs java"><span class="hljs-keyword">public</span> <span class="hljs-keyword">void</span> <span class="hljs-title function_">copyFromLocalFile</span><span class="hljs-params">(Path src, Path dst)</span><br><span class="hljs-keyword">public</span> <span class="hljs-keyword">void</span> <span class="hljs-title function_">moveFromLocalFile</span><span class="hljs-params">(Path[] srcs, Path dst)</span><br><span class="hljs-keyword">public</span> <span class="hljs-keyword">void</span> <span class="hljs-title function_">copyFromLocalFile</span><span class="hljs-params">(<span class="hljs-type">boolean</span> delSrc, Path src, Path dst)</span><br><span class="hljs-keyword">public</span> <span class="hljs-keyword">void</span> <span class="hljs-title function_">copyFromLocalFile</span><span class="hljs-params">(<span class="hljs-type">boolean</span> delSrc, <span class="hljs-type">boolean</span> overwrite,Path[] srcs, Path dst)</span><br><span class="hljs-keyword">public</span> <span class="hljs-keyword">void</span> <span class="hljs-title function_">copyFromLocalFile</span><span class="hljs-params">(<span class="hljs-type">boolean</span> delSrc, <span class="hljs-type">boolean</span> overwrite,Path src, Path dst)</span><br></code></pre></td></tr></table></figure>
<p><strong>参数说明：</strong></p>
<table>
<thead>
<tr>
<th align="left">参数</th>
<th align="left">描述</th>
</tr>
</thead>
<tbody><tr>
<td align="left"><code>boolean delSrc</code></td>
<td align="left">是否删除HDFS路径所在的原数据</td>
</tr>
<tr>
<td align="left"><code>boolean overwrite</code></td>
<td align="left">是否允许覆盖原内容</td>
</tr>
<tr>
<td align="left"><code>Path src / []srcs</code></td>
<td align="left">本地数据路径</td>
</tr>
<tr>
<td align="left"><code>Path dst</code></td>
<td align="left">HDFS目的地路径</td>
</tr>
<tr>
<td align="left">测试代码：</td>
<td align="left"></td>
</tr>
</tbody></table>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><code class="hljs java"><span class="hljs-meta">@Test</span><br><span class="hljs-keyword">public</span> <span class="hljs-keyword">void</span> <span class="hljs-title function_">testCopyFromLocalFile</span><span class="hljs-params">()</span> <span class="hljs-keyword">throws</span> IOException&#123;<br>    <span class="hljs-type">FileSystem</span> <span class="hljs-variable">fs</span> <span class="hljs-operator">=</span> getFileSystem();<br>    <span class="hljs-type">Path</span> <span class="hljs-variable">localFile</span> <span class="hljs-operator">=</span> <span class="hljs-keyword">new</span> <span class="hljs-title class_">Path</span>(<span class="hljs-string">&quot;test.txt&quot;</span>);<br>    <span class="hljs-type">Path</span> <span class="hljs-variable">distPath</span> <span class="hljs-operator">=</span> <span class="hljs-keyword">new</span> <span class="hljs-title class_">Path</span>(<span class="hljs-string">&quot;/uni/testMkdir&quot;</span>);<br>    fs.copyFromLocalFile(localFile, distPath);<br>    fs.close();<br>&#125;<br></code></pre></td></tr></table></figure>

<h2 id="2-3-下载文件"><a href="#2-3-下载文件" class="headerlink" title="2.3 下载文件"></a>2.3 下载文件</h2><p><code>org.apache.hadoop.fs.FileSystem</code> 源码中的相关方法：</p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><code class="hljs java"><span class="hljs-keyword">public</span> <span class="hljs-keyword">void</span> <span class="hljs-title function_">copyToLocalFile</span><span class="hljs-params">(<span class="hljs-type">boolean</span> delSrc, Path src, Path dst)</span><br><span class="hljs-keyword">public</span> <span class="hljs-keyword">void</span> <span class="hljs-title function_">copyToLocalFile</span><span class="hljs-params">(<span class="hljs-type">boolean</span> delSrc, Path src, Path dst,<span class="hljs-type">boolean</span> useRawLocalFileSystem)</span> <span class="hljs-keyword">throws</span> IOException <br><span class="hljs-keyword">public</span> <span class="hljs-keyword">void</span> <span class="hljs-title function_">copyToLocalFile</span><span class="hljs-params">(Path src, Path dst)</span> <span class="hljs-keyword">throws</span> IOException<br></code></pre></td></tr></table></figure>

<p><strong>参数说明：</strong></p>
<table>
<thead>
<tr>
<th align="left">参数</th>
<th align="left">描述</th>
</tr>
</thead>
<tbody><tr>
<td align="left"><code>boolean delSrc</code></td>
<td align="left">是否删除本地路径所在的原数据</td>
</tr>
<tr>
<td align="left"><code>boolean overwrite</code></td>
<td align="left">是否允许覆盖原内容</td>
</tr>
<tr>
<td align="left"><code>Path src / []srcs</code></td>
<td align="left">本地数据路径</td>
</tr>
<tr>
<td align="left"><code>Path dst</code></td>
<td align="left">HDFS目的地路径</td>
</tr>
<tr>
<td align="left"><code>boolean useRawLocalFileSystem</code></td>
<td align="left">是否开启本地文件校验（crc循环冗余校验）</td>
</tr>
</tbody></table>
<h2 id="2-4-删除文件"><a href="#2-4-删除文件" class="headerlink" title="2.4 删除文件"></a>2.4 删除文件</h2><p><code>org.apache.hadoop.fs.FileSystem</code> 源码中的相关方法：</p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><code class="hljs java"><span class="hljs-keyword">public</span> <span class="hljs-type">boolean</span> <span class="hljs-title function_">delete</span><span class="hljs-params">(Path f)</span> <span class="hljs-keyword">throws</span> IOException<br><span class="hljs-keyword">public</span> <span class="hljs-keyword">abstract</span> <span class="hljs-type">boolean</span> <span class="hljs-title function_">delete</span><span class="hljs-params">(Path f, <span class="hljs-type">boolean</span> recursive)</span> <span class="hljs-keyword">throws</span> IOException<br><span class="hljs-keyword">public</span> <span class="hljs-type">boolean</span> <span class="hljs-title function_">deleteOnExit</span><span class="hljs-params">(Path f)</span> <span class="hljs-keyword">throws</span> IOException<br><span class="hljs-keyword">public</span> <span class="hljs-keyword">void</span> <span class="hljs-title function_">deleteSnapshot</span><span class="hljs-params">(Path path, String snapshotName)</span><br></code></pre></td></tr></table></figure>
<p><strong>参数说明：</strong></p>
<table>
<thead>
<tr>
<th align="left">参数</th>
<th align="left">描述</th>
</tr>
</thead>
<tbody><tr>
<td align="left"><code>Path f</code></td>
<td align="left">要删除的路径地址</td>
</tr>
<tr>
<td align="left"><code>boolean recursive</code></td>
<td align="left">表示是否递归删除</td>
</tr>
<tr>
<td align="left"><code>Path dst</code></td>
<td align="left">HDFS目的地路径</td>
</tr>
<tr>
<td align="left"><code>boolean useRawLocalFileSystem</code></td>
<td align="left">是否开启本地文件校验（crc循环冗余校验）</td>
</tr>
</tbody></table>
<p><strong>测试代码：</strong></p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre></td><td class="code"><pre><code class="hljs java"><span class="hljs-meta">@Test</span><br><span class="hljs-keyword">public</span> <span class="hljs-keyword">void</span> <span class="hljs-title function_">testDelete</span><span class="hljs-params">()</span>&#123;<br>    <span class="hljs-type">FileSystem</span> <span class="hljs-variable">fs</span> <span class="hljs-operator">=</span> getFileSystem();<br>    <span class="hljs-keyword">try</span>&#123;<br>        fs.delete(<span class="hljs-keyword">new</span> <span class="hljs-title class_">Path</span>(<span class="hljs-string">&quot;/uni/testMkdir/test.txt&quot;</span>),<span class="hljs-literal">true</span>);<br>        fs.close();<br>    &#125; <span class="hljs-keyword">catch</span> (IOException e) &#123;<br>        e.printStackTrace();<br>    &#125;<br>&#125;<br></code></pre></td></tr></table></figure>

<h2 id="2-5-文件的更名和移动"><a href="#2-5-文件的更名和移动" class="headerlink" title="2.5 文件的更名和移动"></a>2.5 文件的更名和移动</h2><p><code>org.apache.hadoop.fs.FileSystem</code> 源码中的相关方法：</p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><code class="hljs java"><span class="hljs-keyword">public</span> <span class="hljs-keyword">abstract</span> <span class="hljs-type">boolean</span> <span class="hljs-title function_">rename</span><span class="hljs-params">(Path src, Path dst)</span> <span class="hljs-keyword">throws</span> IOException<br></code></pre></td></tr></table></figure>

<p><strong>测试方法：</strong></p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre></td><td class="code"><pre><code class="hljs java"><span class="hljs-meta">@Test</span><br><span class="hljs-keyword">public</span> <span class="hljs-keyword">void</span> <span class="hljs-title function_">testMove</span><span class="hljs-params">()</span>&#123;<br>    <span class="hljs-type">FileSystem</span> <span class="hljs-variable">fs</span> <span class="hljs-operator">=</span> getFileSystem();<br>    <span class="hljs-keyword">try</span>&#123;<br>        fs.rename(<span class="hljs-keyword">new</span> <span class="hljs-title class_">Path</span>(<span class="hljs-string">&quot;/uni/testMkdir&quot;</span>),<span class="hljs-keyword">new</span> <span class="hljs-title class_">Path</span>(<span class="hljs-string">&quot;/uni/MyDir&quot;</span>));<br>        fs.close();<br>    &#125; <span class="hljs-keyword">catch</span> (IOException e) &#123;<br>        e.printStackTrace();<br>    &#125;<br>    fs.close();<br>&#125;<br></code></pre></td></tr></table></figure>
<h2 id="2-6-获取HDFS文件信息"><a href="#2-6-获取HDFS文件信息" class="headerlink" title="2.6 获取HDFS文件信息"></a>2.6 获取HDFS文件信息</h2><p><code>org.apache.hadoop.fs.FileSystem</code> <strong>源码中的相关方法</strong>：</p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><code class="hljs java"><span class="hljs-keyword">public</span> RemoteIterator&lt;LocatedFileStatus&gt; <span class="hljs-title function_">listFiles</span><span class="hljs-params">(<span class="hljs-keyword">final</span> Path f, <span class="hljs-keyword">final</span> <span class="hljs-type">boolean</span> recursive)</span><br></code></pre></td></tr></table></figure>


<p><strong>测试方法:</strong><br><strong>获取HDFS文件信息</strong></p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br></pre></td><td class="code"><pre><code class="hljs java"><span class="hljs-meta">@Test</span><br><span class="hljs-keyword">public</span> <span class="hljs-keyword">void</span> <span class="hljs-title function_">testDetail</span><span class="hljs-params">()</span> <span class="hljs-keyword">throws</span> IOException &#123;<br>    <span class="hljs-type">FileSystem</span> <span class="hljs-variable">fs</span> <span class="hljs-operator">=</span> getFileSystem();<br>    <span class="hljs-comment">// 获取所有文件信息</span><br>    RemoteIterator&lt;LocatedFileStatus&gt; listFiles = fs.listFiles(<span class="hljs-keyword">new</span> <span class="hljs-title class_">Path</span>(<span class="hljs-string">&quot;/&quot;</span>), <span class="hljs-literal">true</span>);<br>    <span class="hljs-keyword">while</span> (listFiles.hasNext()) &#123;<br>        <span class="hljs-type">LocatedFileStatus</span> <span class="hljs-variable">fileStatus</span> <span class="hljs-operator">=</span> listFiles.next();<br><br>        System.out.println(<span class="hljs-string">&quot;===========&quot;</span> + fileStatus.getPath() +<span class="hljs-string">&quot;=========&quot;</span>);<br>        System.out.println(fileStatus.getPermission());<br>        System.out.println(fileStatus.getOwner());<br>        System.out.println(fileStatus.getGroup());<br>        System.out.println(fileStatus.getLen());<br>        System.out.println(fileStatus.getModificationTime());<br>        System.out.println(fileStatus.getReplication());<br>        System.out.println(fileStatus.getBlockSize());<br>        System.out.println(fileStatus.getPath().getName());<br>    &#125;<br>    fs.close();<br>&#125;<br></code></pre></td></tr></table></figure>

<p><strong>获取HDFS文件的块信息：</strong></p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre></td><td class="code"><pre><code class="hljs java"><span class="hljs-meta">@Test</span><br><span class="hljs-keyword">public</span> <span class="hljs-keyword">void</span> <span class="hljs-title function_">testDetail</span><span class="hljs-params">()</span> <span class="hljs-keyword">throws</span> IOException &#123;<br>    <span class="hljs-type">FileSystem</span> <span class="hljs-variable">fs</span> <span class="hljs-operator">=</span> getFileSystem();<br>    <span class="hljs-comment">// 获取所有文件信息</span><br>    RemoteIterator&lt;LocatedFileStatus&gt; listFiles = fs.listFiles(<span class="hljs-keyword">new</span> <span class="hljs-title class_">Path</span>(<span class="hljs-string">&quot;/&quot;</span>), <span class="hljs-literal">true</span>);<br>    <span class="hljs-keyword">while</span> (listFiles.hasNext()) &#123;<br>        <span class="hljs-type">LocatedFileStatus</span> <span class="hljs-variable">fileStatus</span> <span class="hljs-operator">=</span> listFiles.next();<br>        BlockLocation[] blockLocations = fileStatus.getBlockLocations();<br>        System.out.println(Arrays.toString(blockLocations));<br>    &#125;<br>	fs.close();<br>&#125;<br></code></pre></td></tr></table></figure>

<h2 id="2-7-文件与文件夹的判断"><a href="#2-7-文件与文件夹的判断" class="headerlink" title="2.7 文件与文件夹的判断"></a>2.7 文件与文件夹的判断</h2><figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre></td><td class="code"><pre><code class="hljs java"><span class="hljs-meta">@Test</span><br><span class="hljs-keyword">public</span> <span class="hljs-keyword">void</span> <span class="hljs-title function_">testFile</span><span class="hljs-params">()</span> <span class="hljs-keyword">throws</span> IOException &#123;<br>    <span class="hljs-type">FileSystem</span> <span class="hljs-variable">fs</span> <span class="hljs-operator">=</span> getFileSystem();<br>    FileStatus[] listStatus = fs.listStatus(<span class="hljs-keyword">new</span> <span class="hljs-title class_">Path</span>(<span class="hljs-string">&quot;/&quot;</span>));<br>    <span class="hljs-keyword">for</span> (FileStatus status : listStatus) &#123;<br>        <span class="hljs-keyword">if</span>(status.isFile())<br>            System.out.println(<span class="hljs-string">&quot;文件:&quot;</span> + status.getPath().getName());<br>        <span class="hljs-keyword">else</span><br>            System.out.println(<span class="hljs-string">&quot;目录:&quot;</span> + status.getPath().getName());<br>    &#125;<br>    fs.close();<br>&#125;<br></code></pre></td></tr></table></figure>
<h2 id="2-8-HDFS-API-配置参数优先级问题"><a href="#2-8-HDFS-API-配置参数优先级问题" class="headerlink" title="2.8 HDFS - API 配置参数优先级问题"></a>2.8 HDFS - API 配置参数优先级问题</h2><h3 id="2-8-1-通过配置文件"><a href="#2-8-1-通过配置文件" class="headerlink" title="2.8.1 通过配置文件"></a>2.8.1 通过配置文件</h3><p><code>hdfs-site.xml</code></p>
<p>可以将hadoop的配置文件 <code>hdfs-site.xml</code>放到项目里的resources文件夹里，这样API在执行时会按照这个配置来执行。</p>
<h3 id="2-8-2-通过Configuration对象"><a href="#2-8-2-通过Configuration对象" class="headerlink" title="2.8.2 通过Configuration对象"></a>2.8.2 通过Configuration对象</h3><p>除了创建文件外，还可以通过配置<code>Configuration</code>对象来设置参数</p>
<p>这个类是来自包 <code>org.apache.hadoop.conf</code> 的</p>
<p><strong>范例：</strong> 将分区数设置成2</p>
<figure class="highlight java"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><code class="hljs java"><span class="hljs-type">Configuration</span> <span class="hljs-variable">configuration</span> <span class="hljs-operator">=</span> <span class="hljs-keyword">new</span> <span class="hljs-title class_">Configuration</span>();<br>configuration.set(<span class="hljs-string">&quot;dfs.replication&quot;</span>, <span class="hljs-number">2</span>);<br></code></pre></td></tr></table></figure>
<h3 id="2-8-3-优先级问题"><a href="#2-8-3-优先级问题" class="headerlink" title="2.8.3 优先级问题"></a>2.8.3 优先级问题</h3><p>优先级从高到低依次为：</p>
<p><em><strong>API程序的Configuration配置 &gt; 项目资源文件下的配置文件 &gt; 集群中 <code>hdfs-site.xml</code> &gt; 集群中<code>hdfs-default.xml</code></strong></em></p>

                
              </div>
            
            <hr/>
            <div>
              <div class="post-metas my-3">
  
    <div class="post-meta mr-3 d-flex align-items-center">
      <i class="iconfont icon-category"></i>
      

<span class="category-chains">
  
  
    
      <span class="category-chain">
        
  <a href="/categories/%E5%A4%A7%E6%95%B0%E6%8D%AE/" class="category-chain-item">大数据</a>
  
  

      </span>
    
  
</span>

    </div>
  
  
    <div class="post-meta">
      <i class="iconfont icon-tags"></i>
      
        <a href="/tags/HDFS/">#HDFS</a>
      
    </div>
  
</div>


              
  

  <div class="license-box my-3">
    <div class="license-title">
      <div>Hadoop _ HDFS 学习笔记（二）HDFS Java API 环境搭建 _ Java操作HDFS文件系统 _ 多案例</div>
      <div>http://example.com/2022/01/18/Hadoop _ HDFS 学习笔记（二）HDFS Java API 环境搭建 _ Java操作HDFS文件系统 _ 多案例/</div>
    </div>
    <div class="license-meta">
      
        <div class="license-meta-item">
          <div>作者</div>
          <div>John Doe</div>
        </div>
      
      
        <div class="license-meta-item license-meta-date">
          <div>发布于</div>
          <div>2022年1月18日</div>
        </div>
      
      
      <div class="license-meta-item">
        <div>许可协议</div>
        <div>
          
            
            
              <a target="_blank" href="https://creativecommons.org/licenses/by/4.0/">
              <span class="hint--top hint--rounded" aria-label="BY - 署名">
                <i class="iconfont icon-by"></i>
              </span>
              </a>
            
          
        </div>
      </div>
    </div>
    <div class="license-icon iconfont"></div>
  </div>



              
                <div class="post-prevnext my-3">
                  <article class="post-prev col-6">
                    
                    
                      <a href="/2022/01/18/Hadoop%20_%20HDFS%20%E5%AD%A6%E4%B9%A0%E7%AC%94%E8%AE%B0%EF%BC%88%E4%B8%89%EF%BC%89HDFS%20%E8%AF%BB%E5%86%99%E6%B5%81%E7%A8%8B%20_%20NN%E3%80%812NN%E3%80%81DN%20%E5%B7%A5%E4%BD%9C%E6%9C%BA%E5%88%B6%20_%20FsImage%E4%B8%8EEdits%E7%9A%84%E5%90%88%E5%B9%B6/" title="Hadoop _ HDFS 学习笔记（三）HDFS 读写流程 _ NN、2NN、DN 工作机制 _ FsImage与Edits的合并">
                        <i class="iconfont icon-arrowleft"></i>
                        <span class="hidden-mobile">Hadoop _ HDFS 学习笔记（三）HDFS 读写流程 _ NN、2NN、DN 工作机制 _ FsImage与Edits的合并</span>
                        <span class="visible-mobile">上一篇</span>
                      </a>
                    
                  </article>
                  <article class="post-next col-6">
                    
                    
                      <a href="/2022/01/17/Hadoop%20_%20HDFS%E5%AD%A6%E4%B9%A0%E7%AC%94%E8%AE%B0%EF%BC%88%E4%B8%80%EF%BC%89HDFS%E4%BC%98%E7%BC%BA%E7%82%B9%E3%80%81NN+DN+2NN%E7%BB%84%E6%88%90%E6%9E%B6%E6%9E%84%E3%80%81block%E6%96%87%E4%BB%B6%E5%9D%97%20_%20Shell%E6%93%8D%E4%BD%9C%20_%20HDFS%E5%B8%B8%E7%94%A8%E5%91%BD%E4%BB%A4%E6%B1%87%E6%80%BB/" title="Hadoop _ HDFS学习笔记（一）HDFS优缺点、NN+DN+2NN组成架构、block文件块 _ Shell操作 _ HDFS常用命令汇总">
                        <span class="hidden-mobile">Hadoop _ HDFS学习笔记（一）HDFS优缺点、NN+DN+2NN组成架构、block文件块 _ Shell操作 _ HDFS常用命令汇总</span>
                        <span class="visible-mobile">下一篇</span>
                        <i class="iconfont icon-arrowright"></i>
                      </a>
                    
                  </article>
                </div>
              
            </div>

            
          </article>
        </div>
      </div>
    </div>

    <div class="side-col d-none d-lg-block col-lg-2">
      
  <aside class="sidebar" style="margin-left: -1rem">
    <div id="toc">
  <p class="toc-header"><i class="iconfont icon-list"></i>&nbsp;目录</p>
  <div class="toc-body" id="toc-body"></div>
</div>



  </aside>


    </div>
  </div>
</div>





  



  



  



  



  







    

    
      <a id="scroll-top-button" aria-label="TOP" href="#" role="button">
        <i class="iconfont icon-arrowup" aria-hidden="true"></i>
      </a>
    

    
      <div class="modal fade" id="modalSearch" tabindex="-1" role="dialog" aria-labelledby="ModalLabel"
     aria-hidden="true">
  <div class="modal-dialog modal-dialog-scrollable modal-lg" role="document">
    <div class="modal-content">
      <div class="modal-header text-center">
        <h4 class="modal-title w-100 font-weight-bold">搜索</h4>
        <button type="button" id="local-search-close" class="close" data-dismiss="modal" aria-label="Close">
          <span aria-hidden="true">&times;</span>
        </button>
      </div>
      <div class="modal-body mx-3">
        <div class="md-form mb-5">
          <input type="text" id="local-search-input" class="form-control validate">
          <label data-error="x" data-success="v" for="local-search-input">关键词</label>
        </div>
        <div class="list-group" id="local-search-result"></div>
      </div>
    </div>
  </div>
</div>

    

    
  </main>

  <footer>
    <div class="footer-inner">
  
    <div class="footer-content">
       <a href="https://hexo.io" target="_blank" rel="nofollow noopener"><span>Hexo</span></a> <i class="iconfont icon-love"></i> <a href="https://github.com/fluid-dev/hexo-theme-fluid" target="_blank" rel="nofollow noopener"><span>Fluid</span></a> 
    </div>
  
  
  
  
</div>

  </footer>

  <!-- Scripts -->
  
  <script  src="https://lib.baomitu.com/nprogress/0.2.0/nprogress.min.js" ></script>
  <link  rel="stylesheet" href="https://lib.baomitu.com/nprogress/0.2.0/nprogress.min.css" />

  <script>
    NProgress.configure({"showSpinner":false,"trickleSpeed":100})
    NProgress.start()
    window.addEventListener('load', function() {
      NProgress.done();
    })
  </script>


<script  src="https://lib.baomitu.com/jquery/3.6.0/jquery.min.js" ></script>
<script  src="https://lib.baomitu.com/twitter-bootstrap/4.6.1/js/bootstrap.min.js" ></script>
<script  src="/js/events.js" ></script>
<script  src="/js/plugins.js" ></script>


  <script  src="https://lib.baomitu.com/typed.js/2.0.12/typed.min.js" ></script>
  <script>
    (function (window, document) {
      var typing = Fluid.plugins.typing;
      var subtitle = document.getElementById('subtitle');
      if (!subtitle || !typing) {
        return;
      }
      var text = subtitle.getAttribute('data-typed-text');
      
        typing(text);
      
    })(window, document);
  </script>




  
    <script  src="/js/img-lazyload.js" ></script>
  




  
<script>
  Fluid.utils.createScript('https://lib.baomitu.com/tocbot/4.18.2/tocbot.min.js', function() {
    var toc = jQuery('#toc');
    if (toc.length === 0 || !window.tocbot) { return; }
    var boardCtn = jQuery('#board-ctn');
    var boardTop = boardCtn.offset().top;

    window.tocbot.init({
      tocSelector     : '#toc-body',
      contentSelector : '.markdown-body',
      headingSelector : CONFIG.toc.headingSelector || 'h1,h2,h3,h4,h5,h6',
      linkClass       : 'tocbot-link',
      activeLinkClass : 'tocbot-active-link',
      listClass       : 'tocbot-list',
      isCollapsedClass: 'tocbot-is-collapsed',
      collapsibleClass: 'tocbot-is-collapsible',
      collapseDepth   : CONFIG.toc.collapseDepth || 0,
      scrollSmooth    : true,
      headingsOffset  : -boardTop
    });
    if (toc.find('.toc-list-item').length > 0) {
      toc.css('visibility', 'visible');
    }
  });
</script>


  <script src=https://lib.baomitu.com/clipboard.js/2.0.10/clipboard.min.js></script>

  <script>Fluid.plugins.codeWidget();</script>


  
<script>
  Fluid.utils.createScript('https://lib.baomitu.com/anchor-js/4.3.1/anchor.min.js', function() {
    window.anchors.options = {
      placement: CONFIG.anchorjs.placement,
      visible  : CONFIG.anchorjs.visible
    };
    if (CONFIG.anchorjs.icon) {
      window.anchors.options.icon = CONFIG.anchorjs.icon;
    }
    var el = (CONFIG.anchorjs.element || 'h1,h2,h3,h4,h5,h6').split(',');
    var res = [];
    for (var item of el) {
      res.push('.markdown-body > ' + item.trim());
    }
    if (CONFIG.anchorjs.placement === 'left') {
      window.anchors.options.class = 'anchorjs-link-left';
    }
    window.anchors.add(res.join(', '));
  });
</script>


  
<script>
  Fluid.utils.createScript('https://lib.baomitu.com/fancybox/3.5.7/jquery.fancybox.min.js', function() {
    Fluid.plugins.fancyBox();
  });
</script>


  <script>Fluid.plugins.imageCaption();</script>

  <script  src="/js/local-search.js" ></script>





<!-- 主题的启动项，将它保持在最底部 -->
<!-- the boot of the theme, keep it at the bottom -->
<script  src="/js/boot.js" ></script>


  

  <noscript>
    <div class="noscript-warning">博客在允许 JavaScript 运行的环境下浏览效果更佳</div>
  </noscript>
</body>
</html>
